{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import kenlm\n",
    "from tqdm import tqdm\n",
    "import fastText\n",
    "import pandas as pd\n",
    "from bleu import *\n",
    "import torch, os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "#bert classifier\n",
    "\n",
    "from tqdm import trange\n",
    "\n",
    "from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE\n",
    "from pytorch_pretrained_bert.modeling import BertForSequenceClassification, BertConfig, WEIGHTS_NAME, CONFIG_NAME\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "\n",
    "model_cls = BertForSequenceClassification.from_pretrained(\"./bert_classifier/amazon\", num_labels=2)\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
    "\n",
    "model_cls.to('cuda')\n",
    "model_cls.eval()\n",
    "\n",
    "max_seq_len=70\n",
    "sm = torch.nn.Softmax(dim=-1)\n",
    "\n",
    "def evaluate_dev_set(input_sentences, labels, bs=32):\n",
    "    \"\"\"\n",
    "    To evaluate whole dataset and return accuracy\n",
    "    \"\"\"\n",
    "    ids = []\n",
    "    segment_ids = []\n",
    "    input_masks = []\n",
    "    pred_lt = []\n",
    "    for sen in input_sentences:\n",
    "        text_tokens = tokenizer.tokenize(sen)\n",
    "        if len(text_tokens) >= max_seq_len - 2:\n",
    "            text_tokens = text_tokens[:max_seq_len - 3]\n",
    "        tokens = [\"[CLS]\"] + text_tokens + [\"[SEP]\"]\n",
    "        temp_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(temp_ids)\n",
    "        segment_id = [0] * len(temp_ids)\n",
    "        padding = [0] * (max_seq_len - len(temp_ids))\n",
    "\n",
    "        temp_ids += padding\n",
    "        input_mask += padding\n",
    "        segment_id += padding\n",
    "        \n",
    "        ids.append(temp_ids)\n",
    "        input_masks.append(input_mask)\n",
    "        segment_ids.append(segment_id)\n",
    "    \n",
    "    ids = torch.tensor(ids).to('cuda')\n",
    "    segment_ids = torch.tensor(segment_ids).to('cuda')\n",
    "    input_masks = torch.tensor(input_masks).to('cuda')\n",
    "    \n",
    "    steps = len(ids) // bs\n",
    "    \n",
    "    for i in trange(steps+1):\n",
    "        if i == steps:\n",
    "            temp_ids = ids[i * bs : len(ids)]\n",
    "            temp_segment_ids = segment_ids[i * bs: len(ids)]\n",
    "            temp_input_masks = input_masks[i * bs: len(ids)]\n",
    "        else:\n",
    "            temp_ids = ids[i * bs : i * bs + bs]\n",
    "            temp_segment_ids = segment_ids[i * bs: i * bs + bs]\n",
    "            temp_input_masks = input_masks[i * bs: i * bs + bs]\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            preds = sm(model_cls(temp_ids, temp_segment_ids, temp_input_masks))\n",
    "        \n",
    "        #preds = preds.view(-1,bs)\n",
    "        try:\n",
    "            args = torch.argmax(preds, dim=-1)\n",
    "            pred_lt.extend(args.tolist())\n",
    "        except RuntimeError:\n",
    "            pass\n",
    "    accuracy = sum(np.array(pred_lt) == np.array(labels)) / len(labels)\n",
    "    \n",
    "    return accuracy, pred_lt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_pretrained_bert.tokenization_gpt2:loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json from cache at /home/ubuntu/.pytorch_pretrained_bert/f2808208f9bec2320371a9f5f891c184ae0b674ef866b79c58177067d15732dd.1512018be4ba4e8726e41b9145129dc30651ea4fec86aa61f4b9f40bf94eac71\n",
      "INFO:pytorch_pretrained_bert.tokenization_gpt2:loading merges file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt from cache at /home/ubuntu/.pytorch_pretrained_bert/d629f792e430b3c76a1291bb2766b0a047e36fae0588f9dbc1ae51decdff691b.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
      "INFO:pytorch_pretrained_bert.modeling_gpt2:loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin from cache at /home/ubuntu/.pytorch_pretrained_bert/4295d67f022061768f4adc386234dbdb781c814c39662dd1662221c309962c55.778cf36f5c4e5d94c8cd9cefcf2a580c8643570eb327f0d4a1f007fab2acbdf1\n",
      "INFO:pytorch_pretrained_bert.modeling_gpt2:loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json from cache at /home/ubuntu/.pytorch_pretrained_bert/4be02c5697d91738003fb1685c9872f284166aa32e061576bbe6aaeb95649fcf.085d5f6a8e7812ea05ff0e6ed0645ab2e75d80387ad55c1ad9806ee70d272f80\n",
      "INFO:pytorch_pretrained_bert.modeling_gpt2:Model config {\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"n_ctx\": 1024,\n",
      "  \"n_embd\": 768,\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"n_positions\": 1024,\n",
      "  \"vocab_size\": 50257\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pytorch_pretrained_bert import GPT2Tokenizer, GPT2Model, GPT2LMHeadModel\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "lm_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "lm_model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
    "path = os.path.join(os.getcwd(), \"GPT2/amazon_language_model_1.bin\")\n",
    "lm_model_state_dict = torch.load(path)\n",
    "lm_model.load_state_dict(lm_model_state_dict)\n",
    "lm_model.to(device)\n",
    "lm_model.eval()\n",
    "\n",
    "lm_loss = torch.nn.CrossEntropyLoss(ignore_index=-1, reduction='none')\n",
    "\n",
    "\n",
    "def calculate_ppl_gpt2(sentence_batch, bs=16):\n",
    "    # tokenize the sentences\n",
    "    tokenized_ids = [None for i in range(len(sentence_batch))]\n",
    "    ppl = [None for i in range(len(sentence_batch))]\n",
    "    \n",
    "    for i in range(len(sentence_batch)):\n",
    "        tokenized_ids[i] = lm_tokenizer.encode(sentence_batch[i])\n",
    "        \n",
    "    sen_lengths = [len(x) for x in tokenized_ids]\n",
    "    max_sen_length = max(sen_lengths)\n",
    "    \n",
    "    n_batch = len(sentence_batch)\n",
    "    input_ids = np.zeros( shape=(n_batch, max_sen_length), dtype=np.int64)\n",
    "    lm_labels = np.full(shape=(n_batch, max_sen_length), fill_value=-1)\n",
    "    \n",
    "    for i, tokens in enumerate(tokenized_ids):\n",
    "        input_ids[i, :len(tokens)] = tokens\n",
    "        lm_labels[i, :len(tokens)-1] = tokens[1:] \n",
    "    \n",
    "    input_ids = torch.tensor(input_ids)#.to(device)\n",
    "    lm_labels = torch.tensor(lm_labels)#.to(device)\n",
    "    \n",
    "    steps = n_batch // bs\n",
    "    \n",
    "    for i in range(steps+1):\n",
    "        \n",
    "        if i == steps:\n",
    "            temp_input_ids = input_ids[i * bs : n_batch]\n",
    "            temp_lm_labels = lm_labels[i * bs : n_batch]\n",
    "            temp_sen_lengths = sen_lengths[i * bs : n_batch]\n",
    "        else:\n",
    "            temp_input_ids = input_ids[i * bs : i * bs + bs]\n",
    "            temp_lm_labels = lm_labels[i * bs : i * bs + bs]\n",
    "            temp_sen_lengths = sen_lengths[i * bs : i * bs + bs]\n",
    "            \n",
    "        temp_input_ids = temp_input_ids.to('cuda')\n",
    "        temp_lm_labels = temp_lm_labels.to('cuda')\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            lm_pred = lm_model(temp_input_ids)\n",
    "            \n",
    "        loss_val = lm_loss(lm_pred[0].view(-1, lm_pred[0].size(-1)), temp_lm_labels.view(-1))\n",
    "        normalized_loss = loss_val.view(len(temp_input_ids),-1).sum(dim= -1) / torch.tensor(temp_sen_lengths, dtype=torch.float32).to(device)\n",
    "        tmp_ppl = torch.exp(normalized_loss)\n",
    "        ppl[i * bs: i * bs + len(temp_input_ids)] = tmp_ppl.tolist()\n",
    "    \n",
    "    return  ppl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fasttext classifier\n",
    "classifier_model = fastText.load_model('fasttextmodel/amazon_model.bin')\n",
    "\n",
    "#kenlm lm\n",
    "kenlm_lm = kenlm.Model('kenlmmodel/amazon.arpa')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/9 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:05,  2.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:04,  2.97it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:04,  3.10it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.17it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.21it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:03,  3.25it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:02<00:02,  3.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:02,  3.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.35it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:04<00:00,  3.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 11%|█         | 1/9 [00:07<01:02,  7.78s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 22%|██▏       | 2/9 [00:16<00:56,  8.01s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.66it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 33%|███▎      | 3/9 [00:29<00:59,  9.92s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.50it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.50it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 44%|████▍     | 4/9 [00:36<00:46,  9.21s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 56%|█████▌    | 5/9 [00:44<00:35,  8.83s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 67%|██████▋   | 6/9 [00:51<00:25,  8.63s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 78%|███████▊  | 7/9 [00:58<00:16,  8.40s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 89%|████████▉ | 8/9 [01:06<00:08,  8.30s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "100%|██████████| 9/9 [01:14<00:00,  8.22s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('amazon_all_model_prediction_1.csv', header = None)\n",
    "label = 0\n",
    "label_str = '__label__0'\n",
    "\n",
    "list_sentences = df[1:len(df)].values.tolist()\n",
    "\n",
    "list_sentences_source = []\n",
    "list_sentences_human = []\n",
    "\n",
    "for list_sentance in list_sentences:\n",
    "    if(pd.isnull(list_sentance[0])):\n",
    "        list_sentences_source.append(\" \")\n",
    "    else:\n",
    "        list_sentences_source.append(list_sentance[0])\n",
    "    \n",
    "    if(pd.isnull(list_sentance[-1])):\n",
    "        list_sentences_human.append(\" \")\n",
    "    else:\n",
    "        list_sentences_human.append(list_sentance[-1])\n",
    "    \n",
    "\n",
    "matrics1 = []\n",
    "for i in tqdm(range(0, len(list_sentences[0]))):\n",
    "    bleu_s = 0\n",
    "    bleu_r = 0\n",
    "    fasttext_c = 0\n",
    "    kenlm_ppl = 0\n",
    "    gpt2_ppl = 0\n",
    "    \n",
    "    sentences = []\n",
    "    for j in range(0, len(list_sentences)):\n",
    "        if(pd.isnull(list_sentences[j][i])):\n",
    "            sentences.append(\" \")\n",
    "            continue\n",
    "        sentences.append(list_sentences[j][i])\n",
    "        \n",
    "    fasttext_labels = classifier_model.predict(sentences)\n",
    "    \n",
    "    total_sentences = len(sentences)\n",
    "\n",
    "    bleu_s = get_bleu(list_sentences_source, sentences)\n",
    "    bleu_r = get_bleu(list_sentences_human, sentences)\n",
    "\n",
    "    for _, sentence in enumerate(sentences):\n",
    "        if(fasttext_labels[0][_][0]==label_str):\n",
    "            fasttext_c += 1\n",
    "        kenlm_ppl += kenlm_lm.perplexity(sentence)\n",
    "        \n",
    "    labels_list = [label] * len(sentences)\n",
    "\n",
    "    bert_accuracy, pred_label_list = evaluate_dev_set(sentences, labels_list)\n",
    "    ppl_list_gpt2 = calculate_ppl_gpt2(sentences)\n",
    "    \n",
    "    for j in range(0, len(ppl_list_gpt2)):\n",
    "        gpt2_ppl += ppl_list_gpt2[j]\n",
    "\n",
    "    matrics1.append([bleu_s , bleu_r , fasttext_c/total_sentences , kenlm_ppl/total_sentences, bert_accuracy, gpt2_ppl/len(ppl_list_gpt2)])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/9 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:05,  2.77it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:04,  2.99it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:04,  3.12it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.18it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.23it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:03,  3.26it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:02<00:02,  3.29it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.32it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:02,  3.34it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.36it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.37it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.38it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.39it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:04<00:00,  3.40it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.41it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.48it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 11%|█         | 1/9 [00:07<00:58,  7.26s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.64it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 22%|██▏       | 2/9 [00:14<00:49,  7.12s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.76it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 33%|███▎      | 3/9 [00:23<00:47,  7.97s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 44%|████▍     | 4/9 [00:31<00:38,  7.80s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.67it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 56%|█████▌    | 5/9 [00:38<00:31,  7.75s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.75it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 67%|██████▋   | 6/9 [00:46<00:23,  7.70s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.71it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 78%|███████▊  | 7/9 [00:53<00:15,  7.60s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.71it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.63it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.56it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 89%|████████▉ | 8/9 [01:00<00:07,  7.57s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.64it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "100%|██████████| 9/9 [01:08<00:00,  7.57s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('amazon_all_model_prediction_0.csv', header = None)\n",
    "label = 1\n",
    "label_str = '__label__1'\n",
    "\n",
    "list_sentences = df[1:len(df)].values.tolist()\n",
    "\n",
    "list_sentences_source = []\n",
    "list_sentences_human = []\n",
    "\n",
    "for list_sentance in list_sentences:\n",
    "    if(pd.isnull(list_sentance[0])):\n",
    "        list_sentences_source.append(\" \")\n",
    "    else:\n",
    "        list_sentences_source.append(list_sentance[0])\n",
    "    \n",
    "    if(pd.isnull(list_sentance[-1])):\n",
    "        list_sentences_human.append(\" \")\n",
    "    else:\n",
    "        list_sentences_human.append(list_sentance[-1])\n",
    "\n",
    "matrics0 = []\n",
    "for i in tqdm(range(0, len(list_sentences[0]))):\n",
    "    bleu_s = 0\n",
    "    bleu_r = 0\n",
    "    fasttext_c = 0\n",
    "    kenlm_ppl = 0\n",
    "    gpt2_ppl = 0\n",
    "\n",
    "    sentences = []\n",
    "    for j in range(0, len(list_sentences)):\n",
    "        if(pd.isnull(list_sentences[j][i])):\n",
    "            sentences.append(\" \")\n",
    "            continue\n",
    "        sentences.append(list_sentences[j][i])\n",
    "        \n",
    "    fasttext_labels = classifier_model.predict(sentences)\n",
    "    \n",
    "    total_sentences = len(sentences)\n",
    "    \n",
    "    bleu_s = get_bleu(list_sentences_source, sentences)\n",
    "    bleu_r = get_bleu(list_sentences_human, sentences)\n",
    "    \n",
    "    for _, sentence in enumerate(sentences):\n",
    "        if(fasttext_labels[0][_][0]==label_str):\n",
    "            fasttext_c += 1\n",
    "        kenlm_ppl += kenlm_lm.perplexity(sentence)\n",
    "        \n",
    "    labels_list = [label] * len(sentences)\n",
    "    bert_accuracy, pred_label_list = evaluate_dev_set(sentences, labels_list)\n",
    "    \n",
    "    ppl_list_gpt2 = calculate_ppl_gpt2(sentences)\n",
    "\n",
    "    for j in range(0, len(ppl_list_gpt2)):\n",
    "        gpt2_ppl += ppl_list_gpt2[j]\n",
    "        \n",
    "    matrics0.append([bleu_s , bleu_r , fasttext_c/total_sentences , kenlm_ppl/total_sentences, bert_accuracy, gpt2_ppl/len(ppl_list_gpt2)])\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100.0, 67.73886758565439, 0.208, 119.45335778348579, 0.15, 34.96666891670227]\n",
      "[14.976605614659091, 14.845597114754561, 0.908, 19.629643258587993, 0.77, 28.809080837249756]\n",
      "[16.23939249839318, 15.330526739499476, 0.526, 84.87614110764278, 0.464, 131.30051573753357]\n",
      "[16.36063461191676, 15.671692117300404, 0.756, 84.01531375202994, 0.71, 129.24480063438415]\n",
      "[16.17150946519909, 14.016477517569381, 0.542, 33773.05888294231, 0.474, 56.648111888885495]\n",
      "[15.7302883326841, 13.869651456684421, 0.592, 76.56198512211884, 0.526, 40.375796036243436]\n",
      "[72.93740220165328, 55.51956117732476, 0.63, 195.51868619591448, 0.658, 56.72542850255966]\n",
      "[70.95090040766817, 50.624013029408566, 0.608, 407.8507847781964, 0.64, 165.67828907108307]\n",
      "[67.85479371420534, 100.0, 0.424, 2607.4768312100746, 0.452, 76.81063127493859]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None, None, None, None, None, None, None]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[print(i) for i in matrics0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100.0, 73.07703444909669, 0.2, 311.31211800210355, 0.138, 30.924947624206542]\n",
      "[15.506475383131862, 14.616047532987405, 0.754, 21.250957704697296, 0.666, 31.39602951860428]\n",
      "[17.225977804686043, 15.999586392807455, 0.384, 89.60899829181518, 0.298, 128.25660062837602]\n",
      "[16.641866845761655, 15.725102417340914, 0.68, 76.83515055272728, 0.63, 115.7281846280098]\n",
      "[16.138771562049822, 14.869266476333278, 0.47, 33800.62883955456, 0.45, 53.42682637453079]\n",
      "[16.377687723501467, 15.030079763084302, 0.426, 128.17468398820753, 0.516, 43.9817015132904]\n",
      "[74.19292935400931, 58.9965825823184, 0.57, 283.8150743920332, 0.6, 53.614639067173]\n",
      "[70.99148690208703, 54.99056521087671, 0.544, 533.6119836489512, 0.572, 176.37393500709533]\n",
      "[73.05091989442911, 100.0, 0.428, 37938.65602114074, 0.612, 77.09565449380875]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None, None, None, None, None, None, None]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[print(i) for i in matrics1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "matricsavg = (np.array(matrics0)+np.array(matrics1))/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res0 = pd.DataFrame(matrics0, columns=['BLEU_source','BLEU_human','fasttext_classifier','klm_ppl', 'BERT_classifier', 'gpt2_ppl'])\n",
    "df_res1 = pd.DataFrame(matrics1, columns=['BLEU_source','BLEU_human','fasttext_classifier','klm_ppl', 'BERT_classifier', 'gpt2_ppl'])\n",
    "df_resavg = pd.DataFrame(matricsavg, columns=['BLEU_source','BLEU_human','fasttext_classifier','klm_ppl', 'BERT_classifier', 'gpt2_ppl'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_list = df[0:1].values.tolist()\n",
    "#df_res.insert(loc=0, column='GLEU_score', value=gleu_list)\n",
    "df_res0.insert(loc=0, column='model', value=models_list[0])\n",
    "df_res1.insert(loc=0, column='model', value=models_list[0])\n",
    "df_resavg.insert(loc=0, column='model', value=models_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>BLEU_source</th>\n",
       "      <th>BLEU_human</th>\n",
       "      <th>fasttext_classifier</th>\n",
       "      <th>klm_ppl</th>\n",
       "      <th>BERT_classifier</th>\n",
       "      <th>gpt2_ppl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Source</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>67.738868</td>\n",
       "      <td>0.208</td>\n",
       "      <td>119.453358</td>\n",
       "      <td>0.150</td>\n",
       "      <td>34.966669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CROSSALIGNED</td>\n",
       "      <td>14.976606</td>\n",
       "      <td>14.845597</td>\n",
       "      <td>0.908</td>\n",
       "      <td>19.629643</td>\n",
       "      <td>0.770</td>\n",
       "      <td>28.809081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>STYLEEMBEDDING</td>\n",
       "      <td>16.239392</td>\n",
       "      <td>15.330527</td>\n",
       "      <td>0.526</td>\n",
       "      <td>84.876141</td>\n",
       "      <td>0.464</td>\n",
       "      <td>131.300516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MULTIDECODER</td>\n",
       "      <td>16.360635</td>\n",
       "      <td>15.671692</td>\n",
       "      <td>0.756</td>\n",
       "      <td>84.015314</td>\n",
       "      <td>0.710</td>\n",
       "      <td>129.244801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DELETEONLY</td>\n",
       "      <td>16.171509</td>\n",
       "      <td>14.016478</td>\n",
       "      <td>0.542</td>\n",
       "      <td>33773.058883</td>\n",
       "      <td>0.474</td>\n",
       "      <td>56.648112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DELETEANDRETRIEVE</td>\n",
       "      <td>15.730288</td>\n",
       "      <td>13.869651</td>\n",
       "      <td>0.592</td>\n",
       "      <td>76.561985</td>\n",
       "      <td>0.526</td>\n",
       "      <td>40.375796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>BERT_DEL</td>\n",
       "      <td>72.937402</td>\n",
       "      <td>55.519561</td>\n",
       "      <td>0.630</td>\n",
       "      <td>195.518686</td>\n",
       "      <td>0.658</td>\n",
       "      <td>56.725429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>BERT_RET_TFIDF</td>\n",
       "      <td>70.950900</td>\n",
       "      <td>50.624013</td>\n",
       "      <td>0.608</td>\n",
       "      <td>407.850785</td>\n",
       "      <td>0.640</td>\n",
       "      <td>165.678289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>HUMAN</td>\n",
       "      <td>67.854794</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.424</td>\n",
       "      <td>2607.476831</td>\n",
       "      <td>0.452</td>\n",
       "      <td>76.810631</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               model  BLEU_source  BLEU_human  fasttext_classifier  \\\n",
       "0             Source   100.000000   67.738868                0.208   \n",
       "1       CROSSALIGNED    14.976606   14.845597                0.908   \n",
       "2     STYLEEMBEDDING    16.239392   15.330527                0.526   \n",
       "3       MULTIDECODER    16.360635   15.671692                0.756   \n",
       "4         DELETEONLY    16.171509   14.016478                0.542   \n",
       "5  DELETEANDRETRIEVE    15.730288   13.869651                0.592   \n",
       "6           BERT_DEL    72.937402   55.519561                0.630   \n",
       "7     BERT_RET_TFIDF    70.950900   50.624013                0.608   \n",
       "8              HUMAN    67.854794  100.000000                0.424   \n",
       "\n",
       "        klm_ppl  BERT_classifier    gpt2_ppl  \n",
       "0    119.453358            0.150   34.966669  \n",
       "1     19.629643            0.770   28.809081  \n",
       "2     84.876141            0.464  131.300516  \n",
       "3     84.015314            0.710  129.244801  \n",
       "4  33773.058883            0.474   56.648112  \n",
       "5     76.561985            0.526   40.375796  \n",
       "6    195.518686            0.658   56.725429  \n",
       "7    407.850785            0.640  165.678289  \n",
       "8   2607.476831            0.452   76.810631  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_res0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res0.to_csv('matrics/amazon/matrics_amazon_all_model_prediction_0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res1.to_csv('matrics/amazon/matrics_amazon_all_model_prediction_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_resavg.to_csv('matrics/amazon/matrics_amazon_all_model_prediction_avg.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
